{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2b5f7462",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\colorado\\AppData\\Local\\Temp\\ipykernel_12880\\3596654255.py:5: UserWarning: Shapely 2.0 is installed, but because PyGEOS is also installed, GeoPandas will still use PyGEOS by default for now. To force to use and test Shapely 2.0, you have to set the environment variable USE_PYGEOS=0. You can do this before starting the Python process, or in your code before importing geopandas:\n",
      "\n",
      "import os\n",
      "os.environ['USE_PYGEOS'] = '0'\n",
      "import geopandas\n",
      "\n",
      "In a future release, GeoPandas will switch to using Shapely by default. If you are using PyGEOS directly (calling PyGEOS functions on geometries from GeoPandas), this will then stop working and you are encouraged to migrate from PyGEOS to Shapely 2.0 (https://shapely.readthedocs.io/en/latest/migration_pygeos.html).\n",
      "  import geopandas as gpd\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import geopandas as gpd\n",
    "import rioxarray as rxr\n",
    "import rasterio \n",
    "import xarray as xr\n",
    "import earthpy as et\n",
    "from multiprocessing import Pool\n",
    "from parallel_xarray_Senegal import sample_tif\n",
    "from get_dynamic_by_date_Senegal import get_date_df, get_SAVI_date_df\n",
    "import pandas as pd \n",
    "from shapely import geometry\n",
    "import datetime\n",
    "from dask import dataframe as dd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d2d7fc2b",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'gpd' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[2], line 6\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;66;03m# Create a grid of points - 250m spacing \u001b[39;00m\n\u001b[0;32m      2\u001b[0m \n\u001b[0;32m      3\u001b[0m \u001b[38;5;66;03m# Read the shapefile - CHANGE THIS TO SHAPEFILE YOU WANT FISHNET OVER \u001b[39;00m\n\u001b[0;32m      4\u001b[0m \n\u001b[0;32m      5\u001b[0m \u001b[38;5;66;03m# df = gpd.read_file('F:\\\\Senegal_Veg_Model\\\\Static\\\\senegal_boundary\\\\senegal_boundary.shp')\u001b[39;00m\n\u001b[1;32m----> 6\u001b[0m df \u001b[38;5;241m=\u001b[39m \u001b[43mgpd\u001b[49m\u001b[38;5;241m.\u001b[39mread_file(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mF:\u001b[39m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124mSenegal_Veg_Model\u001b[39m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124mStatic\u001b[39m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124mKaffrine.shp\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m      8\u001b[0m \u001b[38;5;66;03m# set crs so you can reproject\u001b[39;00m\n\u001b[0;32m      9\u001b[0m df\u001b[38;5;241m.\u001b[39mcrs \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mEPSG:4326\u001b[39m\u001b[38;5;124m\"\u001b[39m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'gpd' is not defined"
     ]
    }
   ],
   "source": [
    "# Create a grid of points - 250m spacing \n",
    "\n",
    "# Read the shapefile - CHANGE THIS TO SHAPEFILE YOU WANT FISHNET OVER \n",
    "\n",
    "# df = gpd.read_file('F:\\\\Senegal_Veg_Model\\\\Static\\\\senegal_boundary\\\\senegal_boundary.shp')\n",
    "df = gpd.read_file('F:\\\\Senegal_Veg_Model\\\\Static\\\\Kaffrine.shp')\n",
    "\n",
    "# set crs so you can reproject\n",
    "df.crs = \"EPSG:4326\"\n",
    "\n",
    "# Reproject to projected coordinate system\n",
    "df = df.to_crs('EPSG:3857')\n",
    "\n",
    "# 250m spacing - CHANGE TO WHATEVER RESOLUTION YOU WANT YOUR FISHNET\n",
    "spacing = 100\n",
    "\n",
    "# get the bounds\n",
    "xmin, ymin, xmax, ymax = df.total_bounds\n",
    "\n",
    "# pull the x and y coordinates\n",
    "xcoords = [i for i in np.arange(xmin, xmax, spacing)]\n",
    "ycoords = [i for i in np.arange(ymin, ymax, spacing)]\n",
    "\n",
    "pointcoords = np.array(np.meshgrid(xcoords, ycoords)).T.reshape(-1, 2) #A 2D array like [[x1,y1], [x1,y2], ...\n",
    "points = gpd.points_from_xy(x=pointcoords[:,0], y=pointcoords[:,1])\n",
    "grid = gpd.GeoSeries(points, crs=df.crs)\n",
    "grid.name = 'geometry'\n",
    "\n",
    "#If you just want to points inside polygons:\n",
    "gridinside = gpd.sjoin(gpd.GeoDataFrame(grid), df[['geometry']], how=\"inner\")\n",
    "\n",
    "fishnet = gpd.GeoDataFrame(gridinside, columns=['geometry']).set_crs('EPSG:3857')\n",
    "\n",
    "# CHANGE OUTPUT TO WHERE YOU WANT YOUR FISHNET OF POINTS\n",
    "fishnet.to_file('F:\\\\Senegal_Veg_Model\\\\Static\\\\Kaffrine_fishnet100m.shp')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d28ae5a4",
   "metadata": {},
   "source": [
    "## Sample Tiff files with point centroids - Static Data\n",
    "Change all crs projections to epsg:4326"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "11ef3572",
   "metadata": {},
   "source": [
    "### Point Centroids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b2ee3ab7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# open points shapefiles \n",
    "# points shapefiles crs = epsg:3857\n",
    "# CHANGE TO YOUR FISHNET FILE\n",
    "# shp = gpd.read_file('F:\\\\Senegal_Veg_Model\\\\Static\\\\Kaffrine_fishnet100m.shp').to_crs(epsg = 4326)\n",
    "\n",
    "# # get point centroids to put lat and lon in csv\n",
    "\n",
    "# shp['lon'] = shp['geometry'].x\n",
    "# shp['lat'] = shp['geometry'].y\n",
    "\n",
    "# shp.to_csv('F:\\\\Senegal_Veg_Model\\\\Static\\\\Kaffrine_fishnet100m.csv')\n",
    "\n",
    "################################ ONCE YOU HAVE RUN THE ABOVE YOU CAN DO THIS TO ACCESS LAT LON \n",
    "\n",
    "# pull csv of latitude logitude coordinates of points \n",
    "\n",
    "# path = 'F:\\\\Senegal_Veg_Model\\\\Static\\\\senegal_fishnet100m.csv'\n",
    "path = 'F:\\\\Senegal_Veg_Model\\\\Static\\\\Kaffrine_fishnet100m.csv'\n",
    "\n",
    "pts = pd.read_csv(path)\n",
    "\n",
    "lats = list(pts.lat)\n",
    "lons = list(pts.lon)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c24ec839",
   "metadata": {},
   "source": [
    "### LULC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2c7306bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pull LULC raster - crs = epsg:4326\n",
    "lulc = \"F:\\\\Senegal_Veg_Model\\\\Static\\\\Cropland_senegal_30m.tif\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f46978d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# lulc arguments \n",
    "lulc_args = (lulc, lats, lons)\n",
    "lulc = sample_tif(lulc_args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "155edcd9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# make dataframe with column from lulc \n",
    "pts['lulc'] = lulc[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "10374378",
   "metadata": {},
   "outputs": [],
   "source": [
    "crop_pts = pts[pts.lulc == 1]\n",
    "crop_lats = list(crop_pts.lat)\n",
    "crop_lons = list(crop_pts.lon)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8e0d41f",
   "metadata": {},
   "source": [
    "### SoilGrids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0e918501",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pull SoilGrid rasters - crs epsg:7030\n",
    "sg_path = \"F:\\\\Senegal_Veg_Model\\\\Static\\\\soil_textural_tifs\"\n",
    "sg_files = os.listdir(sg_path)\n",
    "# take full profile rasters - calculated as weighted average of soil horizons\n",
    "sg_paths = [os.path.join(sg_path, file) for file in sg_files if \"0_100cm_inv\" in file]\n",
    "topsoil_paths = [os.path.join(sg_path, file) for file in sg_files if \"0_30cm\" in file]\n",
    "subsoil_paths = [os.path.join(sg_path, file) for file in sg_files if \"30_100cm\" in file]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "29243239",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "in main.\n",
      "in pool.\n"
     ]
    }
   ],
   "source": [
    "# soilgrids arguments -full profile\n",
    "sg_args = [(path, crop_lats, crop_lons) for path in sg_paths]\n",
    "\n",
    "arguments = sg_args\n",
    "# PARALLEL \n",
    "if __name__ == '__main__':\n",
    "    print (\"in main.\")\n",
    "    with Pool(14) as pool:\n",
    "        print (\"in pool.\")\n",
    "        sg_result = pool.map(sample_tif, arguments)\n",
    "        pool.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ce879ed4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# make a dataframe from sg results \n",
    "sg_cols = [(pair[0][47:-4]) for pair in sg_result]\n",
    "sg_list = [pair[1] for pair in sg_result]\n",
    "sg_df = pd.DataFrame(sg_list).transpose()\n",
    "sg_df.columns = sg_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "02163f49",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "in main.\n",
      "in pool.\n"
     ]
    }
   ],
   "source": [
    "# soilgrids arguments - topsoil\n",
    "top_args = [(path, crop_lats, crop_lons) for path in topsoil_paths]\n",
    "\n",
    "arguments = top_args\n",
    "# PARALLEL \n",
    "if __name__ == '__main__':\n",
    "    print (\"in main.\")\n",
    "    with Pool(14) as pool:\n",
    "        print (\"in pool.\")\n",
    "        topsoil_result = pool.map(sample_tif, arguments)\n",
    "        pool.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c6bff461",
   "metadata": {},
   "outputs": [],
   "source": [
    "# make a dataframe from sg results \n",
    "top_cols = [(pair[0][47:-4]) for pair in topsoil_result]\n",
    "top_list = [pair[1] for pair in topsoil_result]\n",
    "top_df = pd.DataFrame(top_list).transpose()\n",
    "top_df.columns = top_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "cba114c3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "in main.\n",
      "in pool.\n"
     ]
    }
   ],
   "source": [
    "# soilgrids arguments - topsoil\n",
    "sub_args = [(path, crop_lats, crop_lons) for path in subsoil_paths]\n",
    "\n",
    "arguments = sub_args\n",
    "# PARALLEL \n",
    "if __name__ == '__main__':\n",
    "    print (\"in main.\")\n",
    "    with Pool(14) as pool:\n",
    "        print (\"in pool.\")\n",
    "        subsoil_result = pool.map(sample_tif, arguments)\n",
    "        pool.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "9df5d29c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# make a dataframe from sg results \n",
    "sub_cols = [(pair[0][47:-4]) for pair in subsoil_result]\n",
    "sub_list = [pair[1] for pair in subsoil_result]\n",
    "sub_df = pd.DataFrame(sub_list).transpose()\n",
    "sub_df.columns = sub_cols"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64c18cfa",
   "metadata": {},
   "source": [
    "### Slope"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "bbe8919b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pull slope raster - \n",
    "slope = \"F:\\\\Senegal_Veg_Model\\\\Static\\\\Slope_250_goode.tif\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "259946f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# slope arguments \n",
    "slope_args = (slope, crop_lats, crop_lons)\n",
    "slope = sample_tif(slope_args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "69ef2a44",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\colorado\\AppData\\Local\\Temp\\ipykernel_11940\\2124780989.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  crop_pts['slope'] = slope[1]\n"
     ]
    }
   ],
   "source": [
    "# make dataframe with column from slope \n",
    "crop_pts['slope'] = slope[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "46e425e0",
   "metadata": {},
   "source": [
    "### Elevation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "cc21a986",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pull elevation raster - crs epsg:32636\n",
    "elev = \"F:\\\\Senegal_Veg_Model\\\\Static\\\\DEM_Senegal_mosaic1.tif\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "ffa532de",
   "metadata": {},
   "outputs": [],
   "source": [
    "#  arguments\n",
    "elev_args = (elev, crop_lats, crop_lons)\n",
    "elev = sample_tif(elev_args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "0b6fd436",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\colorado\\AppData\\Local\\Temp\\ipykernel_11940\\1537847963.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  crop_pts['elevation'] = elev[1]\n"
     ]
    }
   ],
   "source": [
    "# make dataframe with column from slope \n",
    "crop_pts['elevation'] = elev[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a4911eb3",
   "metadata": {},
   "source": [
    "### Concatenate Static Together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "b7bf07d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# OUTPUT OF STATIC DATA\n",
    "final_df = pd.concat([crop_pts, sg_df, top_df, sub_df], axis = 1)\n",
    "out_df = final_df[['FID', 'lat', 'lon', 'lulc', 'slope',\n",
    "       'elevation', 'bdod0_100cm_inv_weight_mean', 'cfvo0_100cm_inv_weight_mean', 'clay0_100cm_inv_weight_mean',\n",
    "       'sand0_100cm_inv_weight_mean', 'silt0_100cm_inv_weight_mean', 'soc0_100cm_inv_weight_mean',\n",
    "                  'bdod0_30cm_weight_mean', 'cfvo0_30cm_weight_mean', 'clay0_30cm_weight_mean',\n",
    "       'sand0_30cm_weight_mean', 'silt0_30cm_weight_mean', 'soc0_30cm_weight_mean',\n",
    "                  'bdod30_100cm_weight_mean', 'cfvo30_100cm_weight_mean', 'clay30_100cm_weight_mean',\n",
    "       'sand30_100cm_weight_mean', 'silt30_100cm_weight_mean', 'soc30_100cm_weight_mean']]\n",
    "out_df.to_csv('F:\\\\Senegal_Veg_Model\\\\Static\\\\Kaffrine_crop_static_data100m.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab0292c5",
   "metadata": {},
   "source": [
    "## Sample Tiff files with point centroids - Dynamic Data\n",
    "For each date, sample points and create a new dataframe \n",
    "For each date, pull last 6 months of climate data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7630b619",
   "metadata": {},
   "outputs": [],
   "source": [
    "# sample only points where lulc is crop \n",
    "pts = pd.read_csv(\"F:\\\\Senegal_Veg_Model\\\\Static\\\\Kaffrine_crop_static_data100m.csv\")[['FID', 'lat', 'lon']]\n",
    "\n",
    "lats = list(pts.lat)\n",
    "lons = list(pts.lon)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "45e1afc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pull dates to then get individual dataframes - \n",
    "\n",
    "# pull SAVI files -\n",
    "vi_path = \"F:\\\\Senegal_Veg_Model\\\\Dynamic\\\\EVI_mosaic\"\n",
    "vi_files = os.listdir(vi_path)\n",
    "savi_files = [os.path.join(vi_path, file) for file in vi_files]\n",
    "\n",
    "# pull dates \n",
    "dates =  [datetime.datetime.strptime(file[:8], '%Y%m%d') for file in vi_files]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "07e706fa",
   "metadata": {},
   "source": [
    "### Get dataframe for each date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "250c4e73",
   "metadata": {},
   "outputs": [],
   "source": [
    "# precip data starts at 03/01/2014, so start with dates 64 days after that so they have a full record of climate\n",
    "cutoff = datetime.datetime(2014, 3, 1) + datetime.timedelta(days=64)\n",
    "# cutoff = datetime.datetime(2018, 1, 1)\n",
    "valid_dates = [d for d in dates if d > cutoff]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "588c52e6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-09-26 00:00:00  done.\n",
      "2022-08-08 00:00:00  done.\n",
      "2018-03-28 00:00:00  done.\n",
      "2019-01-03 00:00:00  done.\n",
      "2017-12-08 00:00:00  done.\n",
      "2019-10-27 00:00:00  done.\n",
      "2018-05-08 00:00:00  done.\n",
      "2021-09-28 00:00:00  done.\n",
      "2022-04-25 00:00:00  done.\n",
      "2017-05-21 00:00:00  done.\n",
      "2015-09-05 00:00:00  done.\n",
      "2014-05-13 00:00:00  done.\n",
      "2015-06-01 00:00:00  done.\n",
      "2016-01-18 00:00:00  done.\n",
      "2017-09-01 00:00:00  done.\n",
      "2015-03-20 00:00:00  done.\n",
      "2017-04-10 00:00:00  done.\n",
      "2018-02-03 00:00:00  done.\n",
      "2022-11-04 00:00:00  done.\n",
      "2017-01-15 00:00:00  done.\n",
      "2022-11-02 00:00:00  done.\n",
      "2022-06-21 00:00:00  done.\n",
      "2019-12-28 00:00:00  done.\n",
      "2022-09-17 00:00:00  done.\n",
      "2018-02-17 00:00:00  done.\n",
      "2017-07-24 00:00:00  done.\n",
      "2021-07-28 00:00:00  done.\n",
      "2020-10-20 00:00:00  done.\n",
      "2021-10-07 00:00:00  done.\n",
      "2015-07-05 00:00:00  done.\n",
      "2018-04-29 00:00:00  done.\n",
      "2022-12-20 00:00:00  done.\n",
      "2017-09-17 00:00:00  done.\n",
      "2017-10-30 00:00:00  done.\n",
      "2019-12-30 00:00:00  done.\n",
      "2020-07-16 00:00:00  done.\n",
      "2018-04-13 00:00:00  done.\n",
      "2016-08-15 00:00:00  done.\n",
      "2016-12-05 00:00:00  done.\n",
      "2016-04-09 00:00:00  done.\n",
      "2021-12-10 00:00:00  done.\n",
      "2016-06-26 00:00:00  done.\n",
      "2016-12-28 00:00:00  done.\n",
      "2016-05-09 00:00:00  done.\n",
      "2015-01-08 00:00:00  done.\n",
      "2018-06-18 00:00:00  done.\n",
      "2016-04-02 00:00:00  done.\n",
      "2022-03-02 00:00:00  done.\n",
      "2014-12-30 00:00:00  done.\n",
      "2022-04-26 00:00:00  done.\n",
      "2018-03-30 00:00:00  done.\n",
      "2022-01-21 00:00:00  done.\n",
      "2022-12-13 00:00:00  done.\n",
      "2017-05-30 00:00:00  done.\n",
      "2021-01-01 00:00:00  done.\n",
      "2017-06-15 00:00:00  done.\n",
      "2018-04-22 00:00:00  done.\n",
      "2018-01-07 00:00:00  done.\n",
      "2020-05-13 00:00:00  done.\n",
      "2022-07-16 00:00:00  done.\n",
      "2017-03-18 00:00:00  done.\n",
      "2019-09-25 00:00:00  done.\n",
      "2015-03-31 00:00:00  done.\n",
      "2016-02-14 00:00:00  done.\n",
      "2017-09-10 00:00:00  done.\n",
      "2016-05-02 00:00:00  done.\n",
      "2022-01-19 00:00:00  done.\n",
      "2019-05-02 00:00:00  done.\n",
      "2016-01-27 00:00:00  done.\n",
      "2018-09-13 00:00:00  done.\n",
      "2020-06-05 00:00:00  done.\n",
      "2021-01-10 00:00:00  done.\n",
      "2020-10-29 00:00:00  done.\n",
      "2014-08-10 00:00:00  done.\n",
      "2019-10-02 00:00:00  done.\n",
      "2021-05-16 00:00:00  done.\n",
      "2016-11-03 00:00:00  done.\n",
      "2021-09-07 00:00:00  done.\n",
      "2017-10-28 00:00:00  done.\n",
      "2015-01-17 00:00:00  done.\n",
      "2022-03-18 00:00:00  done.\n",
      "2017-12-24 00:00:00  done.\n",
      "2022-05-13 00:00:00  done.\n",
      "2019-11-10 00:00:00  done.\n",
      "2015-12-10 00:00:00  done.\n",
      "2015-02-16 00:00:00  done.\n",
      "2014-05-29 00:00:00  done.\n",
      "2018-01-25 00:00:00  done.\n",
      "2019-05-20 00:00:00  done.\n",
      "2018-04-06 00:00:00  done.\n",
      "2022-04-01 00:00:00  done.\n",
      "2022-11-05 00:00:00  done.\n",
      "2022-11-27 00:00:00  done.\n",
      "2014-07-16 00:00:00  done.\n",
      "2018-03-14 00:00:00  done.\n",
      "2022-12-05 00:00:00  done.\n",
      "2019-05-18 00:00:00  done.\n",
      "2022-03-09 00:00:00  done.\n",
      "2017-06-06 00:00:00  done.\n",
      "2019-03-08 00:00:00  done.\n",
      "2020-06-23 00:00:00  done.\n",
      "2019-03-31 00:00:00  done.\n",
      "2020-03-19 00:00:00  done.\n",
      "2021-01-08 00:00:00  done.\n",
      "2016-09-07 00:00:00  done.\n",
      "2016-03-06 00:00:00  done.\n",
      "2022-08-22 00:00:00  done.\n",
      "2022-02-12 00:00:00  done.\n",
      "2022-01-05 00:00:00  done.\n",
      "2016-01-11 00:00:00  done.\n",
      "2019-08-01 00:00:00  done.\n",
      "2014-11-28 00:00:00  done.\n",
      "2021-10-14 00:00:00  done.\n",
      "2016-05-11 00:00:00  done.\n",
      "2019-12-14 00:00:00  done.\n",
      "2021-04-23 00:00:00  done.\n",
      "2017-09-28 00:00:00  done.\n",
      "2014-12-25 00:00:00  done.\n",
      "2016-02-19 00:00:00  done.\n",
      "2021-06-19 00:00:00  done.\n",
      "2022-11-10 00:00:00  done.\n",
      "2016-10-11 00:00:00  done.\n",
      "2019-03-10 00:00:00  done.\n",
      "2022-12-31 00:00:00  done.\n",
      "2022-02-04 00:00:00  done.\n",
      "2014-10-27 00:00:00  done.\n",
      "2017-04-19 00:00:00  done.\n",
      "2017-10-21 00:00:00  done.\n",
      "2020-10-04 00:00:00  done.\n",
      "2022-11-12 00:00:00  done.\n",
      "2015-11-24 00:00:00  done.\n",
      "2014-11-07 00:00:00  done.\n",
      "2018-08-21 00:00:00  done.\n",
      "2019-03-26 00:00:00  done.\n",
      "2015-11-26 00:00:00  done.\n",
      "2021-12-27 00:00:00  done.\n",
      "2018-03-12 00:00:00  done.\n",
      "2019-05-04 00:00:00  done.\n",
      "2020-07-02 00:00:00  done.\n",
      "2019-09-23 00:00:00  done.\n",
      "2021-08-13 00:00:00  done.\n",
      "2015-02-27 00:00:00  done.\n",
      "2022-04-16 00:00:00  done.\n",
      "2016-06-19 00:00:00  done.\n",
      "2020-03-12 00:00:00  done.\n",
      "2018-12-20 00:00:00  done.\n",
      "2017-12-17 00:00:00  done.\n",
      "2017-04-21 00:00:00  done.\n",
      "2018-12-11 00:00:00  done.\n",
      "2020-03-26 00:00:00  done.\n",
      "2021-10-23 00:00:00  done.\n",
      "2016-05-04 00:00:00  done.\n",
      "2016-11-01 00:00:00  done.\n",
      "2022-10-28 00:00:00  done.\n",
      "2019-12-05 00:00:00  done.\n",
      "2016-07-28 00:00:00  done.\n",
      "2014-11-21 00:00:00  done.\n",
      "2018-06-11 00:00:00  done.\n",
      "2021-04-30 00:00:00  done.\n",
      "2021-11-08 00:00:00  done.\n",
      "2014-05-06 00:00:00  done.\n",
      "2020-08-01 00:00:00  done.\n",
      "2020-10-06 00:00:00  done.\n",
      "2022-03-16 00:00:00  done.\n",
      "2022-05-26 00:00:00  done.\n",
      "2016-03-17 00:00:00  done.\n",
      "2021-11-24 00:00:00  done.\n",
      "2017-05-14 00:00:00  done.\n",
      "2014-10-20 00:00:00  done.\n",
      "2021-01-26 00:00:00  done.\n",
      "2018-10-17 00:00:00  done.\n",
      "2021-02-02 00:00:00  done.\n",
      "2016-07-30 00:00:00  done.\n",
      "2022-11-26 00:00:00  done.\n",
      "2021-05-09 00:00:00  done.\n",
      "2018-10-22 00:00:00  done.\n",
      "2017-01-31 00:00:00  done.\n",
      "2021-07-03 00:00:00  done.\n",
      "2020-01-13 00:00:00  done.\n",
      "2015-10-30 00:00:00  done.\n",
      "2020-03-28 00:00:00  done.\n",
      "2021-02-27 00:00:00  done.\n",
      "2017-02-16 00:00:00  done.\n",
      "2022-06-29 00:00:00  done.\n",
      "2022-05-12 00:00:00  done.\n",
      "2017-10-05 00:00:00  done.\n",
      "2015-01-15 00:00:00  done.\n",
      "2015-05-25 00:00:00  done.\n",
      "2021-06-08 00:00:00  done.\n",
      "2022-11-03 00:00:00  done.\n",
      "2015-07-28 00:00:00  done.\n",
      "2017-01-13 00:00:00  done.\n",
      "2015-06-19 00:00:00  done.\n",
      "2021-09-14 00:00:00  done.\n",
      "2021-11-05 00:00:00  done.\n",
      "2022-01-13 00:00:00  done.\n",
      "2017-10-03 00:00:00  done.\n",
      "2017-12-01 00:00:00  done.\n",
      "2022-04-18 00:00:00  done.\n",
      "2022-08-16 00:00:00  done.\n",
      "2019-01-28 00:00:00  done.\n",
      "2017-05-07 00:00:00  done.\n",
      "2022-11-19 00:00:00  done.\n",
      "2021-05-02 00:00:00  done.\n",
      "2019-02-04 00:00:00  done.\n",
      "2017-02-07 00:00:00  done.\n",
      "2016-06-03 00:00:00  done.\n",
      "2020-09-27 00:00:00  done.\n",
      "2015-03-22 00:00:00  done.\n",
      "2019-01-19 00:00:00  done.\n",
      "2022-10-04 00:00:00  done.\n",
      "2017-02-23 00:00:00  done.\n",
      "2020-03-17 00:00:00  done.\n",
      "2022-06-22 00:00:00  done.\n",
      "2018-07-11 00:00:00  done.\n",
      "2022-03-26 00:00:00  done.\n",
      "2022-01-03 00:00:00  done.\n",
      "2021-09-12 00:00:00  done.\n",
      "2022-02-19 00:00:00  done.\n",
      "2016-05-27 00:00:00  done.\n",
      "2014-05-31 00:00:00  done.\n",
      "2022-04-17 00:00:00  done.\n",
      "2021-11-26 00:00:00  done.\n",
      "2016-02-21 00:00:00  done.\n",
      "2015-05-09 00:00:00  done.\n",
      "2021-12-18 00:00:00  done.\n",
      "2020-10-22 00:00:00  done.\n",
      "2014-10-04 00:00:00  done.\n",
      "2018-12-18 00:00:00  done.\n",
      "2022-02-20 00:00:00  done.\n",
      "2019-11-12 00:00:00  done.\n",
      "2014-09-18 00:00:00  done.\n",
      "2015-01-26 00:00:00  done.\n",
      "2019-11-21 00:00:00  done.\n",
      "2017-01-22 00:00:00  done.\n",
      "2018-02-26 00:00:00  done.\n",
      "2016-03-08 00:00:00  done.\n",
      "2021-03-22 00:00:00  done.\n",
      "2018-05-26 00:00:00  done.\n",
      "2022-01-20 00:00:00  done.\n",
      "2022-09-08 00:00:00  done.\n",
      "2018-03-05 00:00:00  done.\n",
      "2019-04-02 00:00:00  done.\n",
      "2015-08-27 00:00:00  done.\n",
      "2016-11-26 00:00:00  done.\n",
      "2022-02-28 00:00:00  done.\n",
      "2016-06-05 00:00:00  done.\n",
      "2022-11-11 00:00:00  done.\n",
      "2021-02-18 00:00:00  done.\n",
      "2022-04-08 00:00:00  done.\n",
      "2018-10-24 00:00:00  done.\n",
      "2022-05-05 00:00:00  done.\n",
      "2016-02-28 00:00:00  done.\n",
      "2019-03-24 00:00:00  done.\n",
      "2020-11-28 00:00:00  done.\n",
      "2022-02-03 00:00:00  done.\n",
      "2022-05-10 00:00:00  done.\n",
      "2015-01-01 00:00:00  done.\n",
      "2018-10-31 00:00:00  done.\n",
      "2020-10-13 00:00:00  done.\n",
      "2017-08-02 00:00:00  done.\n",
      "2016-03-15 00:00:00  done.\n",
      "2020-07-07 00:00:00  done.\n",
      "2019-08-08 00:00:00  done.\n",
      "2019-10-20 00:00:00  done.\n",
      "2014-05-20 00:00:00  done.\n",
      "2021-12-28 00:00:00  done.\n",
      "2019-01-21 00:00:00  done.\n",
      "2014-09-09 00:00:00  done.\n",
      "2022-12-15 00:00:00  done.\n",
      "2015-02-25 00:00:00  done.\n",
      "2019-11-19 00:00:00  done.\n",
      "2022-06-12 00:00:00  done.\n",
      "2019-07-30 00:00:00  done.\n",
      "2016-10-02 00:00:00  done.\n",
      "2020-11-12 00:00:00  done.\n",
      "2019-02-06 00:00:00  done.\n",
      "2019-09-09 00:00:00  done.\n",
      "2020-05-06 00:00:00  done.\n",
      "2017-06-08 00:00:00  done.\n",
      "2014-09-02 00:00:00  done.\n",
      "2021-03-06 00:00:00  done.\n",
      "2021-06-03 00:00:00  done.\n",
      "2021-12-19 00:00:00  done.\n",
      "2015-07-03 00:00:00  done.\n",
      "2020-02-25 00:00:00  done.\n",
      "2019-04-25 00:00:00  done.\n",
      "2015-10-09 00:00:00  done.\n",
      "2021-01-24 00:00:00  done.\n",
      "2021-12-11 00:00:00  done.\n",
      "2022-02-06 00:00:00  done.\n",
      "2017-04-12 00:00:00  done.\n",
      "2016-04-25 00:00:00  done.\n",
      "2016-04-18 00:00:00  done.\n",
      "2014-06-23 00:00:00  done.\n",
      "2018-01-23 00:00:00  done.\n",
      "2022-05-19 00:00:00  done.\n",
      "2022-11-20 00:00:00  done.\n",
      "2019-07-21 00:00:00  done.\n",
      "2022-12-07 00:00:00  done.\n",
      "2022-06-04 00:00:00  done.\n",
      "2015-04-23 00:00:00  done.\n",
      "2022-09-07 00:00:00  done.\n",
      "2019-06-21 00:00:00  done.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-07-15 00:00:00  done.\n",
      "2021-06-01 00:00:00  done.\n",
      "2021-03-29 00:00:00  done.\n",
      "2016-07-14 00:00:00  done.\n",
      "2019-11-28 00:00:00  done.\n",
      "2020-04-27 00:00:00  done.\n",
      "2014-11-30 00:00:00  done.\n",
      "2017-03-09 00:00:00  done.\n",
      "2015-03-06 00:00:00  done.\n",
      "2022-11-18 00:00:00  done.\n",
      "2018-07-20 00:00:00  done.\n",
      "2014-10-11 00:00:00  done.\n",
      "2016-03-31 00:00:00  done.\n",
      "2019-06-19 00:00:00  done.\n",
      "2014-11-14 00:00:00  done.\n",
      "2022-03-08 00:00:00  done.\n",
      "2018-10-06 00:00:00  done.\n",
      "2015-06-10 00:00:00  done.\n",
      "2017-10-14 00:00:00  done.\n",
      "2022-05-21 00:00:00  done.\n",
      "2017-01-06 00:00:00  done.\n",
      "2022-03-23 00:00:00  done.\n",
      "2017-11-20 00:00:00  done.\n",
      "2022-02-22 00:00:00  done.\n",
      "2022-06-05 00:00:00  done.\n",
      "2016-04-16 00:00:00  done.\n",
      "2021-12-25 00:00:00  done.\n",
      "2022-03-07 00:00:00  done.\n",
      "2015-09-14 00:00:00  done.\n",
      "2022-01-29 00:00:00  done.\n",
      "2022-09-24 00:00:00  done.\n",
      "2014-06-14 00:00:00  done.\n",
      "2016-11-19 00:00:00  done.\n",
      "2017-12-15 00:00:00  done.\n",
      "2014-10-06 00:00:00  done.\n",
      "2022-11-28 00:00:00  done.\n",
      "2021-05-25 00:00:00  done.\n",
      "2017-01-20 00:00:00  done.\n",
      "2019-04-18 00:00:00  done.\n",
      "2015-03-29 00:00:00  done.\n",
      "2020-01-29 00:00:00  done.\n",
      "2021-12-09 00:00:00  done.\n",
      "2019-11-26 00:00:00  done.\n",
      "2020-07-09 00:00:00  done.\n",
      "2022-10-01 00:00:00  done.\n",
      "2021-03-15 00:00:00  done.\n",
      "2014-07-25 00:00:00  done.\n",
      "2017-03-11 00:00:00  done.\n",
      "2020-05-04 00:00:00  done.\n",
      "2015-11-10 00:00:00  done.\n",
      "2014-07-18 00:00:00  done.\n",
      "2022-03-01 00:00:00  done.\n",
      "2021-03-20 00:00:00  done.\n",
      "2019-08-24 00:00:00  done.\n",
      "2022-04-09 00:00:00  done.\n",
      "2021-08-11 00:00:00  done.\n",
      "2016-02-05 00:00:00  done.\n",
      "2018-04-08 00:00:00  done.\n",
      "2015-04-07 00:00:00  done.\n",
      "2022-10-18 00:00:00  done.\n",
      "2017-11-22 00:00:00  done.\n",
      "2022-06-13 00:00:00  done.\n",
      "2022-08-01 00:00:00  done.\n",
      "2022-10-02 00:00:00  done.\n",
      "2017-02-21 00:00:00  done.\n",
      "2020-05-15 00:00:00  done.\n",
      "2019-06-14 00:00:00  done.\n",
      "2017-05-23 00:00:00  done.\n",
      "2017-04-26 00:00:00  done.\n",
      "2016-08-31 00:00:00  done.\n",
      "2017-12-31 00:00:00  done.\n",
      "2019-03-17 00:00:00  done.\n",
      "2014-05-22 00:00:00  done.\n",
      "2015-12-19 00:00:00  done.\n",
      "2021-04-05 00:00:00  done.\n",
      "2019-12-07 00:00:00  done.\n",
      "2022-08-14 00:00:00  done.\n",
      "2018-05-24 00:00:00  done.\n",
      "2021-12-17 00:00:00  done.\n",
      "2022-01-04 00:00:00  done.\n",
      "2014-10-13 00:00:00  done.\n",
      "2020-01-31 00:00:00  done.\n",
      "2016-10-27 00:00:00  done.\n",
      "2020-05-29 00:00:00  done.\n",
      "2016-12-14 00:00:00  done.\n",
      "2015-08-29 00:00:00  done.\n",
      "2022-02-05 00:00:00  done.\n",
      "2017-04-28 00:00:00  done.\n",
      "2018-05-17 00:00:00  done.\n",
      "2019-02-20 00:00:00  done.\n",
      "2020-04-29 00:00:00  done.\n",
      "2022-04-24 00:00:00  done.\n",
      "2020-10-27 00:00:00  done.\n",
      "2020-12-07 00:00:00  done.\n",
      "2022-12-06 00:00:00  done.\n",
      "2022-05-20 00:00:00  done.\n",
      "2019-02-22 00:00:00  done.\n",
      "2019-05-29 00:00:00  done.\n",
      "2021-12-12 00:00:00  done.\n",
      "2022-04-02 00:00:00  done.\n",
      "2019-03-15 00:00:00  done.\n",
      "2020-02-09 00:00:00  done.\n",
      "2020-06-07 00:00:00  done.\n",
      "2019-02-13 00:00:00  done.\n",
      "2015-09-21 00:00:00  done.\n",
      "2015-11-08 00:00:00  done.\n",
      "2019-07-23 00:00:00  done.\n",
      "2015-04-05 00:00:00  done.\n",
      "2015-10-25 00:00:00  done.\n",
      "2017-10-12 00:00:00  done.\n",
      "2016-12-12 00:00:00  done.\n",
      "2019-10-11 00:00:00  done.\n",
      "2014-11-12 00:00:00  done.\n",
      "2022-10-11 00:00:00  done.\n",
      "2021-06-10 00:00:00  done.\n",
      "2020-05-20 00:00:00  done.\n",
      "2019-06-12 00:00:00  done.\n",
      "2018-07-27 00:00:00  done.\n",
      "2021-08-27 00:00:00  done.\n",
      "2022-07-14 00:00:00  done.\n",
      "2019-02-11 00:00:00  done.\n",
      "2022-09-16 00:00:00  done.\n",
      "2021-11-22 00:00:00  done.\n",
      "2022-02-13 00:00:00  done.\n",
      "2022-12-22 00:00:00  done.\n",
      "2020-11-14 00:00:00  done.\n",
      "2022-08-07 00:00:00  done.\n",
      "2017-03-04 00:00:00  done.\n",
      "2016-12-03 00:00:00  done.\n",
      "2022-10-19 00:00:00  done.\n",
      "2015-03-13 00:00:00  done.\n",
      "2018-09-15 00:00:00  done.\n",
      "2022-01-02 00:00:00  done.\n",
      "2016-02-03 00:00:00  done.\n",
      "2022-04-19 00:00:00  done.\n",
      "2015-05-07 00:00:00  done.\n",
      "2019-10-18 00:00:00  done.\n",
      "2021-07-26 00:00:00  done.\n",
      "2021-04-21 00:00:00  done.\n",
      "2022-03-25 00:00:00  done.\n",
      "2022-12-12 00:00:00  done.\n",
      "2020-12-23 00:00:00  done.\n",
      "2017-03-02 00:00:00  done.\n",
      "2017-03-20 00:00:00  done.\n",
      "2022-01-12 00:00:00  done.\n",
      "2015-12-17 00:00:00  done.\n",
      "2020-11-21 00:00:00  done.\n",
      "2019-05-13 00:00:00  done.\n",
      "2018-08-12 00:00:00  done.\n",
      "2016-08-08 00:00:00  done.\n",
      "2015-05-18 00:00:00  done.\n",
      "2021-03-31 00:00:00  done.\n",
      "2015-02-09 00:00:00  done.\n",
      "2017-04-05 00:00:00  done.\n",
      "2016-10-09 00:00:00  done.\n",
      "2016-07-05 00:00:00  done.\n",
      "2016-06-21 00:00:00  done.\n",
      "2020-01-08 00:00:00  done.\n",
      "2016-09-23 00:00:00  done.\n",
      "2022-12-21 00:00:00  done.\n",
      "2014-08-01 00:00:00  done.\n",
      "2018-06-02 00:00:00  done.\n",
      "2022-02-14 00:00:00  done.\n",
      "2016-01-20 00:00:00  done.\n",
      "2019-03-01 00:00:00  done.\n",
      "2022-03-10 00:00:00  done.\n",
      "2017-02-14 00:00:00  done.\n",
      "2019-01-12 00:00:00  done.\n",
      "2016-11-12 00:00:00  done.\n",
      "2022-04-03 00:00:00  done.\n",
      "2022-09-15 00:00:00  done.\n",
      "2022-12-14 00:00:00  done.\n",
      "2014-06-16 00:00:00  done.\n",
      "2015-11-01 00:00:00  done.\n",
      "2021-02-25 00:00:00  done.\n",
      "2021-04-07 00:00:00  done.\n",
      "2014-10-29 00:00:00  done.\n",
      "2018-12-09 00:00:00  done.\n",
      "2017-01-04 00:00:00  done.\n",
      "2014-12-09 00:00:00  done.\n",
      "2020-11-23 00:00:00  done.\n",
      "2021-03-13 00:00:00  done.\n",
      "2020-06-16 00:00:00  done.\n",
      "2017-09-19 00:00:00  done.\n",
      "2022-04-10 00:00:00  done.\n",
      "2018-03-07 00:00:00  done.\n",
      "2018-07-13 00:00:00  done.\n",
      "2015-12-03 00:00:00  done.\n",
      "2014-05-15 00:00:00  done.\n",
      "2022-06-03 00:00:00  done.\n",
      "2022-08-23 00:00:00  done.\n",
      "2017-05-05 00:00:00  done.\n",
      "2021-05-18 00:00:00  done.\n",
      "2017-01-29 00:00:00  done.\n",
      "2020-06-14 00:00:00  done.\n",
      "2018-04-24 00:00:00  done.\n",
      "2018-12-02 00:00:00  done.\n",
      "2022-05-02 00:00:00  done.\n",
      "2018-03-23 00:00:00  done.\n",
      "2014-06-21 00:00:00  done.\n",
      "2017-09-03 00:00:00  done.\n",
      "2021-09-30 00:00:00  done.\n",
      "2018-06-25 00:00:00  done.\n",
      "2014-11-05 00:00:00  done.\n",
      "2019-01-05 00:00:00  done.\n",
      "2015-06-26 00:00:00  done.\n",
      "2022-07-06 00:00:00  done.\n",
      "2022-01-18 00:00:00  done.\n",
      "2022-02-27 00:00:00  done.\n",
      "2017-08-27 00:00:00  done.\n",
      "2015-10-16 00:00:00  done.\n",
      "2020-05-22 00:00:00  done.\n",
      "2020-03-10 00:00:00  done.\n",
      "2017-10-19 00:00:00  done.\n",
      "2015-11-15 00:00:00  done.\n",
      "2016-08-22 00:00:00  done.\n",
      "2021-06-17 00:00:00  done.\n",
      "2016-04-23 00:00:00  done.\n",
      "2021-09-21 00:00:00  done.\n",
      "2020-04-04 00:00:00  done.\n",
      "2014-09-27 00:00:00  done.\n",
      "2016-10-18 00:00:00  done.\n",
      "2022-10-27 00:00:00  done.\n",
      "2018-06-09 00:00:00  done.\n",
      "2019-07-07 00:00:00  done.\n",
      "2021-02-11 00:00:00  done.\n",
      "2018-11-09 00:00:00  done.\n",
      "2021-10-16 00:00:00  done.\n",
      "2016-12-19 00:00:00  done.\n",
      "2021-11-10 00:00:00  done.\n",
      "2020-04-11 00:00:00  done.\n",
      "2020-01-06 00:00:00  done.\n",
      "2021-12-03 00:00:00  done.\n",
      "2022-01-27 00:00:00  done.\n",
      "2022-02-21 00:00:00  done.\n",
      "2022-03-24 00:00:00  done.\n",
      "2019-05-27 00:00:00  done.\n",
      "2020-12-09 00:00:00  done.\n",
      "2022-02-11 00:00:00  done.\n",
      "2018-12-27 00:00:00  done.\n",
      "2022-05-11 00:00:00  done.\n",
      "2017-06-13 00:00:00  done.\n",
      "2019-05-11 00:00:00  done.\n",
      "2019-04-11 00:00:00  done.\n",
      "2016-01-29 00:00:00  done.\n",
      "2022-06-14 00:00:00  done.\n",
      "2020-12-16 00:00:00  done.\n",
      "2017-12-06 00:00:00  done.\n",
      "2022-11-21 00:00:00  done.\n",
      "2014-08-26 00:00:00  done.\n",
      "2019-12-23 00:00:00  done.\n",
      "2017-11-13 00:00:00  done.\n",
      "2020-03-03 00:00:00  done.\n",
      "2020-10-11 00:00:00  done.\n",
      "2018-05-01 00:00:00  done.\n",
      "2022-12-28 00:00:00  done.\n",
      "2015-02-02 00:00:00  done.\n",
      "2019-11-05 00:00:00  done.\n",
      "2018-10-15 00:00:00  done.\n",
      "2022-12-23 00:00:00  done.\n",
      "2018-01-18 00:00:00  done.\n",
      "2019-06-28 00:00:00  done.\n",
      "2015-04-16 00:00:00  done.\n",
      "2017-11-04 00:00:00  done.\n",
      "2020-02-07 00:00:00  done.\n",
      "2022-10-12 00:00:00  done.\n",
      "2017-03-27 00:00:00  done.\n",
      "2014-06-05 00:00:00  done.\n",
      "2016-12-30 00:00:00  done.\n",
      "2018-02-08 00:00:00  done.\n",
      "2017-07-08 00:00:00  done.\n",
      "2021-04-16 00:00:00  done.\n",
      "2015-08-04 00:00:00  done.\n",
      "2016-07-07 00:00:00  done.\n",
      "2020-04-20 00:00:00  done.\n",
      "2021-08-29 00:00:00  done.\n",
      "2018-02-19 00:00:00  done.\n",
      "2022-11-13 00:00:00  done.\n",
      "2014-07-23 00:00:00  done.\n",
      "2020-11-30 00:00:00  done.\n",
      "2017-11-15 00:00:00  done.\n",
      "2022-11-29 00:00:00  done.\n",
      "2022-04-11 00:00:00  done.\n",
      "2021-11-01 00:00:00  done.\n",
      "2022-04-27 00:00:00  done.\n",
      "2016-06-28 00:00:00  done.\n",
      "2017-07-17 00:00:00  done.\n",
      "2014-09-11 00:00:00  done.\n",
      "2019-04-09 00:00:00  done.\n",
      "2022-05-04 00:00:00  done.\n",
      "2018-02-10 00:00:00  done.\n",
      "2015-01-31 00:00:00  done.\n",
      "2022-07-07 00:00:00  done.\n",
      "2018-04-15 00:00:00  done.\n",
      "2019-06-05 00:00:00  done.\n",
      "2017-09-12 00:00:00  done.\n",
      "2021-04-14 00:00:00  done.\n",
      "2018-05-10 00:00:00  done.\n",
      "2018-01-16 00:00:00  done.\n",
      "2021-11-17 00:00:00  done.\n",
      "2022-10-26 00:00:00  done.\n",
      "2015-05-23 00:00:00  done.\n",
      "2018-03-21 00:00:00  done.\n",
      "2020-12-30 00:00:00  done.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2014-08-17 00:00:00  done.\n",
      "2019-04-16 00:00:00  done.\n",
      "2018-01-09 00:00:00  done.\n",
      "2022-06-30 00:00:00  done.\n",
      "2017-03-25 00:00:00  done.\n",
      "2015-05-02 00:00:00  done.\n",
      "2014-12-14 00:00:00  done.\n",
      "2022-01-28 00:00:00  done.\n",
      "2019-07-14 00:00:00  done.\n",
      "2016-05-20 00:00:00  done.\n",
      "2021-12-04 00:00:00  done.\n",
      "2016-09-25 00:00:00  done.\n",
      "2017-04-03 00:00:00  done.\n",
      "2015-02-18 00:00:00  done.\n",
      "2019-04-27 00:00:00  done.\n",
      "2016-05-18 00:00:00  done.\n",
      "2020-04-02 00:00:00  done.\n",
      "2018-12-04 00:00:00  done.\n"
     ]
    }
   ],
   "source": [
    "### Get SAVI values sampled for each date - then merged to the full dataframe eventually \n",
    "# # datewise arguments \n",
    "args = [(date, pts) for date in valid_dates]\n",
    "\n",
    "# arguments = args_tur\n",
    "# # PARALLEL \n",
    "# if __name__ == '__main__':\n",
    "#     print (\"in main.\")\n",
    "#     with Pool(10) as pool:\n",
    "#         print (\"in pool.\")\n",
    "#         date_result_t = pool.map(get_SAVI_date_df, arguments)\n",
    "#         pool.close()\n",
    "\n",
    "# or \n",
    "date_result = []\n",
    "for arg in args: \n",
    "    date_result.append(get_SAVI_date_df(arg))\n",
    "    print (arg[0], \" done.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "d320ebb8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# make a dataframe from date results \n",
    "dynamic_SAVI = pd.concat(date_result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "3566ca89",
   "metadata": {},
   "outputs": [],
   "source": [
    "# put into dask dataframe to handle size\n",
    "from dask import dataframe as dd\n",
    "dyn_dd = dd.from_pandas(dynamic_SAVI, npartitions = 400)\n",
    "# send to parquet file storage \n",
    "name_function = lambda x: f\"kaffrine_dyn_EVI-{x}.parquet\"\n",
    "dyn_dd.to_parquet('F:\\\\Senegal_Veg_Model\\\\parquet', name_function=name_function)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5f2cfb64",
   "metadata": {},
   "source": [
    "## Get zscore + climate dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "53e58835",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-09-26 00:00:00  done.\n",
      "2022-08-08 00:00:00  done.\n",
      "2018-03-28 00:00:00  done.\n",
      "2019-01-03 00:00:00  done.\n",
      "2017-12-08 00:00:00  done.\n",
      "2019-10-27 00:00:00  done.\n",
      "2018-05-08 00:00:00  done.\n",
      "2021-09-28 00:00:00  done.\n",
      "2022-04-25 00:00:00  done.\n",
      "2017-05-21 00:00:00  done.\n",
      "2015-09-05 00:00:00  done.\n",
      "2014-05-13 00:00:00  done.\n",
      "2015-06-01 00:00:00  done.\n",
      "2016-01-18 00:00:00  done.\n",
      "2017-09-01 00:00:00  done.\n",
      "2015-03-20 00:00:00  done.\n",
      "2017-04-10 00:00:00  done.\n",
      "2018-02-03 00:00:00  done.\n",
      "2022-11-04 00:00:00  done.\n",
      "2017-01-15 00:00:00  done.\n",
      "2022-11-02 00:00:00  done.\n",
      "2022-06-21 00:00:00  done.\n",
      "2019-12-28 00:00:00  done.\n",
      "2022-09-17 00:00:00  done.\n",
      "2018-02-17 00:00:00  done.\n",
      "2017-07-24 00:00:00  done.\n",
      "2021-07-28 00:00:00  done.\n",
      "2020-10-20 00:00:00  done.\n",
      "2021-10-07 00:00:00  done.\n",
      "2015-07-05 00:00:00  done.\n",
      "2018-04-29 00:00:00  done.\n",
      "2022-12-20 00:00:00  done.\n",
      "2017-09-17 00:00:00  done.\n",
      "2017-10-30 00:00:00  done.\n",
      "2019-12-30 00:00:00  done.\n",
      "2020-07-16 00:00:00  done.\n",
      "2018-04-13 00:00:00  done.\n",
      "2016-08-15 00:00:00  done.\n",
      "2016-12-05 00:00:00  done.\n",
      "2016-04-09 00:00:00  done.\n",
      "2021-12-10 00:00:00  done.\n",
      "2016-06-26 00:00:00  done.\n",
      "2016-12-28 00:00:00  done.\n",
      "2016-05-09 00:00:00  done.\n",
      "2015-01-08 00:00:00  done.\n",
      "2018-06-18 00:00:00  done.\n",
      "2016-04-02 00:00:00  done.\n",
      "2022-03-02 00:00:00  done.\n",
      "2014-12-30 00:00:00  done.\n",
      "2022-04-26 00:00:00  done.\n",
      "2018-03-30 00:00:00  done.\n",
      "2022-01-21 00:00:00  done.\n",
      "2022-12-13 00:00:00  done.\n",
      "2017-05-30 00:00:00  done.\n",
      "2021-01-01 00:00:00  done.\n",
      "2017-06-15 00:00:00  done.\n",
      "2018-04-22 00:00:00  done.\n",
      "2018-01-07 00:00:00  done.\n",
      "2020-05-13 00:00:00  done.\n",
      "2022-07-16 00:00:00  done.\n",
      "2017-03-18 00:00:00  done.\n",
      "2019-09-25 00:00:00  done.\n",
      "2015-03-31 00:00:00  done.\n",
      "2016-02-14 00:00:00  done.\n",
      "2017-09-10 00:00:00  done.\n",
      "2016-05-02 00:00:00  done.\n",
      "2022-01-19 00:00:00  done.\n",
      "2019-05-02 00:00:00  done.\n",
      "2016-01-27 00:00:00  done.\n",
      "2018-09-13 00:00:00  done.\n",
      "2020-06-05 00:00:00  done.\n",
      "2021-01-10 00:00:00  done.\n",
      "2020-10-29 00:00:00  done.\n",
      "2014-08-10 00:00:00  done.\n",
      "2019-10-02 00:00:00  done.\n",
      "2021-05-16 00:00:00  done.\n",
      "2016-11-03 00:00:00  done.\n",
      "2021-09-07 00:00:00  done.\n",
      "2017-10-28 00:00:00  done.\n",
      "2015-01-17 00:00:00  done.\n",
      "2022-03-18 00:00:00  done.\n",
      "2017-12-24 00:00:00  done.\n",
      "2022-05-13 00:00:00  done.\n",
      "2019-11-10 00:00:00  done.\n",
      "2015-12-10 00:00:00  done.\n",
      "2015-02-16 00:00:00  done.\n",
      "2014-05-29 00:00:00  done.\n",
      "2018-01-25 00:00:00  done.\n",
      "2019-05-20 00:00:00  done.\n",
      "2018-04-06 00:00:00  done.\n",
      "2022-04-01 00:00:00  done.\n",
      "2022-11-05 00:00:00  done.\n",
      "2022-11-27 00:00:00  done.\n",
      "2014-07-16 00:00:00  done.\n",
      "2018-03-14 00:00:00  done.\n",
      "2022-12-05 00:00:00  done.\n",
      "2019-05-18 00:00:00  done.\n",
      "2022-03-09 00:00:00  done.\n",
      "2017-06-06 00:00:00  done.\n",
      "2019-03-08 00:00:00  done.\n",
      "2020-06-23 00:00:00  done.\n",
      "2019-03-31 00:00:00  done.\n",
      "2020-03-19 00:00:00  done.\n",
      "2021-01-08 00:00:00  done.\n",
      "2016-09-07 00:00:00  done.\n",
      "2016-03-06 00:00:00  done.\n",
      "2022-08-22 00:00:00  done.\n",
      "2022-02-12 00:00:00  done.\n",
      "2022-01-05 00:00:00  done.\n",
      "2016-01-11 00:00:00  done.\n",
      "2019-08-01 00:00:00  done.\n",
      "2014-11-28 00:00:00  done.\n",
      "2021-10-14 00:00:00  done.\n",
      "2016-05-11 00:00:00  done.\n",
      "2019-12-14 00:00:00  done.\n",
      "2021-04-23 00:00:00  done.\n",
      "2017-09-28 00:00:00  done.\n",
      "2014-12-25 00:00:00  done.\n",
      "2016-02-19 00:00:00  done.\n",
      "2021-06-19 00:00:00  done.\n",
      "2022-11-10 00:00:00  done.\n",
      "2016-10-11 00:00:00  done.\n",
      "2019-03-10 00:00:00  done.\n",
      "2022-12-31 00:00:00  done.\n",
      "2022-02-04 00:00:00  done.\n",
      "2014-10-27 00:00:00  done.\n",
      "2017-04-19 00:00:00  done.\n",
      "2017-10-21 00:00:00  done.\n",
      "2020-10-04 00:00:00  done.\n",
      "2022-11-12 00:00:00  done.\n",
      "2015-11-24 00:00:00  done.\n",
      "2014-11-07 00:00:00  done.\n",
      "2018-08-21 00:00:00  done.\n",
      "2019-03-26 00:00:00  done.\n",
      "2015-11-26 00:00:00  done.\n",
      "2021-12-27 00:00:00  done.\n",
      "2018-03-12 00:00:00  done.\n",
      "2019-05-04 00:00:00  done.\n",
      "2020-07-02 00:00:00  done.\n",
      "2019-09-23 00:00:00  done.\n",
      "2021-08-13 00:00:00  done.\n",
      "2015-02-27 00:00:00  done.\n",
      "2022-04-16 00:00:00  done.\n",
      "2016-06-19 00:00:00  done.\n",
      "2020-03-12 00:00:00  done.\n",
      "2018-12-20 00:00:00  done.\n",
      "2017-12-17 00:00:00  done.\n",
      "2017-04-21 00:00:00  done.\n",
      "2018-12-11 00:00:00  done.\n",
      "2020-03-26 00:00:00  done.\n",
      "2021-10-23 00:00:00  done.\n",
      "2016-05-04 00:00:00  done.\n",
      "2016-11-01 00:00:00  done.\n",
      "2022-10-28 00:00:00  done.\n",
      "2019-12-05 00:00:00  done.\n",
      "2016-07-28 00:00:00  done.\n",
      "2014-11-21 00:00:00  done.\n",
      "2018-06-11 00:00:00  done.\n",
      "2021-04-30 00:00:00  done.\n",
      "2021-11-08 00:00:00  done.\n",
      "2014-05-06 00:00:00  done.\n",
      "2020-08-01 00:00:00  done.\n",
      "2020-10-06 00:00:00  done.\n",
      "2022-03-16 00:00:00  done.\n",
      "2022-05-26 00:00:00  done.\n",
      "2016-03-17 00:00:00  done.\n",
      "2021-11-24 00:00:00  done.\n",
      "2017-05-14 00:00:00  done.\n",
      "2014-10-20 00:00:00  done.\n",
      "2021-01-26 00:00:00  done.\n",
      "2018-10-17 00:00:00  done.\n",
      "2021-02-02 00:00:00  done.\n",
      "2016-07-30 00:00:00  done.\n",
      "2022-11-26 00:00:00  done.\n",
      "2021-05-09 00:00:00  done.\n",
      "2018-10-22 00:00:00  done.\n",
      "2017-01-31 00:00:00  done.\n",
      "2021-07-03 00:00:00  done.\n",
      "2020-01-13 00:00:00  done.\n",
      "2015-10-30 00:00:00  done.\n",
      "2020-03-28 00:00:00  done.\n",
      "2021-02-27 00:00:00  done.\n",
      "2017-02-16 00:00:00  done.\n",
      "2022-06-29 00:00:00  done.\n",
      "2022-05-12 00:00:00  done.\n",
      "2017-10-05 00:00:00  done.\n",
      "2015-01-15 00:00:00  done.\n",
      "2015-05-25 00:00:00  done.\n",
      "2021-06-08 00:00:00  done.\n",
      "2022-11-03 00:00:00  done.\n",
      "2015-07-28 00:00:00  done.\n",
      "2017-01-13 00:00:00  done.\n",
      "2015-06-19 00:00:00  done.\n",
      "2021-09-14 00:00:00  done.\n",
      "2021-11-05 00:00:00  done.\n",
      "2022-01-13 00:00:00  done.\n",
      "2017-10-03 00:00:00  done.\n",
      "2017-12-01 00:00:00  done.\n",
      "2022-04-18 00:00:00  done.\n",
      "2022-08-16 00:00:00  done.\n",
      "2019-01-28 00:00:00  done.\n",
      "2017-05-07 00:00:00  done.\n",
      "2022-11-19 00:00:00  done.\n",
      "2021-05-02 00:00:00  done.\n",
      "2019-02-04 00:00:00  done.\n",
      "2017-02-07 00:00:00  done.\n",
      "2016-06-03 00:00:00  done.\n",
      "2020-09-27 00:00:00  done.\n",
      "2015-03-22 00:00:00  done.\n",
      "2019-01-19 00:00:00  done.\n",
      "2022-10-04 00:00:00  done.\n",
      "2017-02-23 00:00:00  done.\n",
      "2020-03-17 00:00:00  done.\n",
      "2022-06-22 00:00:00  done.\n",
      "2018-07-11 00:00:00  done.\n",
      "2022-03-26 00:00:00  done.\n",
      "2022-01-03 00:00:00  done.\n",
      "2021-09-12 00:00:00  done.\n",
      "2022-02-19 00:00:00  done.\n",
      "2016-05-27 00:00:00  done.\n",
      "2014-05-31 00:00:00  done.\n",
      "2022-04-17 00:00:00  done.\n",
      "2021-11-26 00:00:00  done.\n",
      "2016-02-21 00:00:00  done.\n",
      "2015-05-09 00:00:00  done.\n",
      "2021-12-18 00:00:00  done.\n",
      "2020-10-22 00:00:00  done.\n",
      "2014-10-04 00:00:00  done.\n",
      "2018-12-18 00:00:00  done.\n",
      "2022-02-20 00:00:00  done.\n",
      "2019-11-12 00:00:00  done.\n",
      "2014-09-18 00:00:00  done.\n",
      "2015-01-26 00:00:00  done.\n",
      "2019-11-21 00:00:00  done.\n",
      "2017-01-22 00:00:00  done.\n",
      "2018-02-26 00:00:00  done.\n",
      "2016-03-08 00:00:00  done.\n",
      "2021-03-22 00:00:00  done.\n",
      "2018-05-26 00:00:00  done.\n",
      "2022-01-20 00:00:00  done.\n",
      "2022-09-08 00:00:00  done.\n",
      "2018-03-05 00:00:00  done.\n",
      "2019-04-02 00:00:00  done.\n",
      "2015-08-27 00:00:00  done.\n",
      "2016-11-26 00:00:00  done.\n",
      "2022-02-28 00:00:00  done.\n",
      "2016-06-05 00:00:00  done.\n",
      "2022-11-11 00:00:00  done.\n",
      "2021-02-18 00:00:00  done.\n",
      "2022-04-08 00:00:00  done.\n",
      "2018-10-24 00:00:00  done.\n",
      "2022-05-05 00:00:00  done.\n",
      "2016-02-28 00:00:00  done.\n",
      "2019-03-24 00:00:00  done.\n",
      "2020-11-28 00:00:00  done.\n",
      "2022-02-03 00:00:00  done.\n",
      "2022-05-10 00:00:00  done.\n",
      "2015-01-01 00:00:00  done.\n",
      "2018-10-31 00:00:00  done.\n",
      "2020-10-13 00:00:00  done.\n",
      "2017-08-02 00:00:00  done.\n",
      "2016-03-15 00:00:00  done.\n",
      "2020-07-07 00:00:00  done.\n",
      "2019-08-08 00:00:00  done.\n",
      "2019-10-20 00:00:00  done.\n",
      "2014-05-20 00:00:00  done.\n",
      "2021-12-28 00:00:00  done.\n",
      "2019-01-21 00:00:00  done.\n",
      "2014-09-09 00:00:00  done.\n",
      "2022-12-15 00:00:00  done.\n",
      "2015-02-25 00:00:00  done.\n",
      "2019-11-19 00:00:00  done.\n",
      "2022-06-12 00:00:00  done.\n",
      "2019-07-30 00:00:00  done.\n",
      "2016-10-02 00:00:00  done.\n",
      "2020-11-12 00:00:00  done.\n",
      "2019-02-06 00:00:00  done.\n",
      "2019-09-09 00:00:00  done.\n",
      "2020-05-06 00:00:00  done.\n",
      "2017-06-08 00:00:00  done.\n",
      "2014-09-02 00:00:00  done.\n",
      "2021-03-06 00:00:00  done.\n",
      "2021-06-03 00:00:00  done.\n",
      "2021-12-19 00:00:00  done.\n",
      "2015-07-03 00:00:00  done.\n",
      "2020-02-25 00:00:00  done.\n",
      "2019-04-25 00:00:00  done.\n",
      "2015-10-09 00:00:00  done.\n",
      "2021-01-24 00:00:00  done.\n",
      "2021-12-11 00:00:00  done.\n",
      "2022-02-06 00:00:00  done.\n",
      "2017-04-12 00:00:00  done.\n",
      "2016-04-25 00:00:00  done.\n",
      "2016-04-18 00:00:00  done.\n",
      "2014-06-23 00:00:00  done.\n",
      "2018-01-23 00:00:00  done.\n",
      "2022-05-19 00:00:00  done.\n",
      "2022-11-20 00:00:00  done.\n",
      "2019-07-21 00:00:00  done.\n",
      "2022-12-07 00:00:00  done.\n",
      "2022-06-04 00:00:00  done.\n",
      "2015-04-23 00:00:00  done.\n",
      "2022-09-07 00:00:00  done.\n",
      "2019-06-21 00:00:00  done.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-07-15 00:00:00  done.\n",
      "2021-06-01 00:00:00  done.\n",
      "2021-03-29 00:00:00  done.\n",
      "2016-07-14 00:00:00  done.\n",
      "2019-11-28 00:00:00  done.\n",
      "2020-04-27 00:00:00  done.\n",
      "2014-11-30 00:00:00  done.\n",
      "2017-03-09 00:00:00  done.\n",
      "2015-03-06 00:00:00  done.\n",
      "2022-11-18 00:00:00  done.\n",
      "2018-07-20 00:00:00  done.\n",
      "2014-10-11 00:00:00  done.\n",
      "2016-03-31 00:00:00  done.\n",
      "2019-06-19 00:00:00  done.\n",
      "2014-11-14 00:00:00  done.\n",
      "2022-03-08 00:00:00  done.\n",
      "2018-10-06 00:00:00  done.\n",
      "2015-06-10 00:00:00  done.\n",
      "2017-10-14 00:00:00  done.\n",
      "2022-05-21 00:00:00  done.\n",
      "2017-01-06 00:00:00  done.\n",
      "2022-03-23 00:00:00  done.\n",
      "2017-11-20 00:00:00  done.\n",
      "2022-02-22 00:00:00  done.\n",
      "2022-06-05 00:00:00  done.\n",
      "2016-04-16 00:00:00  done.\n",
      "2021-12-25 00:00:00  done.\n",
      "2022-03-07 00:00:00  done.\n",
      "2015-09-14 00:00:00  done.\n",
      "2022-01-29 00:00:00  done.\n",
      "2022-09-24 00:00:00  done.\n",
      "2014-06-14 00:00:00  done.\n",
      "2016-11-19 00:00:00  done.\n",
      "2017-12-15 00:00:00  done.\n",
      "2014-10-06 00:00:00  done.\n",
      "2022-11-28 00:00:00  done.\n",
      "2021-05-25 00:00:00  done.\n",
      "2017-01-20 00:00:00  done.\n",
      "2019-04-18 00:00:00  done.\n",
      "2015-03-29 00:00:00  done.\n",
      "2020-01-29 00:00:00  done.\n",
      "2021-12-09 00:00:00  done.\n",
      "2019-11-26 00:00:00  done.\n",
      "2020-07-09 00:00:00  done.\n",
      "2022-10-01 00:00:00  done.\n",
      "2021-03-15 00:00:00  done.\n",
      "2014-07-25 00:00:00  done.\n",
      "2017-03-11 00:00:00  done.\n",
      "2020-05-04 00:00:00  done.\n",
      "2015-11-10 00:00:00  done.\n",
      "2014-07-18 00:00:00  done.\n",
      "2022-03-01 00:00:00  done.\n",
      "2021-03-20 00:00:00  done.\n",
      "2019-08-24 00:00:00  done.\n",
      "2022-04-09 00:00:00  done.\n",
      "2021-08-11 00:00:00  done.\n",
      "2016-02-05 00:00:00  done.\n",
      "2018-04-08 00:00:00  done.\n",
      "2015-04-07 00:00:00  done.\n",
      "2022-10-18 00:00:00  done.\n",
      "2017-11-22 00:00:00  done.\n",
      "2022-06-13 00:00:00  done.\n",
      "2022-08-01 00:00:00  done.\n",
      "2022-10-02 00:00:00  done.\n",
      "2017-02-21 00:00:00  done.\n",
      "2020-05-15 00:00:00  done.\n",
      "2019-06-14 00:00:00  done.\n",
      "2017-05-23 00:00:00  done.\n",
      "2017-04-26 00:00:00  done.\n",
      "2016-08-31 00:00:00  done.\n",
      "2017-12-31 00:00:00  done.\n",
      "2019-03-17 00:00:00  done.\n",
      "2014-05-22 00:00:00  done.\n",
      "2015-12-19 00:00:00  done.\n",
      "2021-04-05 00:00:00  done.\n",
      "2019-12-07 00:00:00  done.\n",
      "2022-08-14 00:00:00  done.\n",
      "2018-05-24 00:00:00  done.\n",
      "2021-12-17 00:00:00  done.\n",
      "2022-01-04 00:00:00  done.\n",
      "2014-10-13 00:00:00  done.\n",
      "2020-01-31 00:00:00  done.\n",
      "2016-10-27 00:00:00  done.\n",
      "2020-05-29 00:00:00  done.\n",
      "2016-12-14 00:00:00  done.\n",
      "2015-08-29 00:00:00  done.\n",
      "2022-02-05 00:00:00  done.\n",
      "2017-04-28 00:00:00  done.\n",
      "2018-05-17 00:00:00  done.\n",
      "2019-02-20 00:00:00  done.\n",
      "2020-04-29 00:00:00  done.\n",
      "2022-04-24 00:00:00  done.\n",
      "2020-10-27 00:00:00  done.\n",
      "2020-12-07 00:00:00  done.\n",
      "2022-12-06 00:00:00  done.\n",
      "2022-05-20 00:00:00  done.\n",
      "2019-02-22 00:00:00  done.\n",
      "2019-05-29 00:00:00  done.\n",
      "2021-12-12 00:00:00  done.\n",
      "2022-04-02 00:00:00  done.\n",
      "2019-03-15 00:00:00  done.\n",
      "2020-02-09 00:00:00  done.\n",
      "2020-06-07 00:00:00  done.\n",
      "2019-02-13 00:00:00  done.\n",
      "2015-09-21 00:00:00  done.\n",
      "2015-11-08 00:00:00  done.\n",
      "2019-07-23 00:00:00  done.\n",
      "2015-04-05 00:00:00  done.\n",
      "2015-10-25 00:00:00  done.\n",
      "2017-10-12 00:00:00  done.\n",
      "2016-12-12 00:00:00  done.\n",
      "2019-10-11 00:00:00  done.\n",
      "2014-11-12 00:00:00  done.\n",
      "2022-10-11 00:00:00  done.\n",
      "2021-06-10 00:00:00  done.\n",
      "2020-05-20 00:00:00  done.\n",
      "2019-06-12 00:00:00  done.\n",
      "2018-07-27 00:00:00  done.\n",
      "2021-08-27 00:00:00  done.\n",
      "2022-07-14 00:00:00  done.\n",
      "2019-02-11 00:00:00  done.\n",
      "2022-09-16 00:00:00  done.\n",
      "2021-11-22 00:00:00  done.\n",
      "2022-02-13 00:00:00  done.\n",
      "2022-12-22 00:00:00  done.\n",
      "2020-11-14 00:00:00  done.\n",
      "2022-08-07 00:00:00  done.\n",
      "2017-03-04 00:00:00  done.\n",
      "2016-12-03 00:00:00  done.\n",
      "2022-10-19 00:00:00  done.\n",
      "2015-03-13 00:00:00  done.\n",
      "2018-09-15 00:00:00  done.\n",
      "2022-01-02 00:00:00  done.\n",
      "2016-02-03 00:00:00  done.\n",
      "2022-04-19 00:00:00  done.\n",
      "2015-05-07 00:00:00  done.\n",
      "2019-10-18 00:00:00  done.\n",
      "2021-07-26 00:00:00  done.\n",
      "2021-04-21 00:00:00  done.\n",
      "2022-03-25 00:00:00  done.\n",
      "2022-12-12 00:00:00  done.\n",
      "2020-12-23 00:00:00  done.\n",
      "2017-03-02 00:00:00  done.\n",
      "2017-03-20 00:00:00  done.\n",
      "2022-01-12 00:00:00  done.\n",
      "2015-12-17 00:00:00  done.\n",
      "2020-11-21 00:00:00  done.\n",
      "2019-05-13 00:00:00  done.\n",
      "2018-08-12 00:00:00  done.\n",
      "2016-08-08 00:00:00  done.\n",
      "2015-05-18 00:00:00  done.\n",
      "2021-03-31 00:00:00  done.\n",
      "2015-02-09 00:00:00  done.\n",
      "2017-04-05 00:00:00  done.\n",
      "2016-10-09 00:00:00  done.\n",
      "2016-07-05 00:00:00  done.\n",
      "2016-06-21 00:00:00  done.\n",
      "2020-01-08 00:00:00  done.\n",
      "2016-09-23 00:00:00  done.\n",
      "2022-12-21 00:00:00  done.\n",
      "2014-08-01 00:00:00  done.\n",
      "2018-06-02 00:00:00  done.\n",
      "2022-02-14 00:00:00  done.\n",
      "2016-01-20 00:00:00  done.\n",
      "2019-03-01 00:00:00  done.\n",
      "2022-03-10 00:00:00  done.\n",
      "2017-02-14 00:00:00  done.\n",
      "2019-01-12 00:00:00  done.\n",
      "2016-11-12 00:00:00  done.\n",
      "2022-04-03 00:00:00  done.\n",
      "2022-09-15 00:00:00  done.\n",
      "2022-12-14 00:00:00  done.\n",
      "2014-06-16 00:00:00  done.\n",
      "2015-11-01 00:00:00  done.\n",
      "2021-02-25 00:00:00  done.\n",
      "2021-04-07 00:00:00  done.\n",
      "2014-10-29 00:00:00  done.\n",
      "2018-12-09 00:00:00  done.\n",
      "2017-01-04 00:00:00  done.\n",
      "2014-12-09 00:00:00  done.\n",
      "2020-11-23 00:00:00  done.\n",
      "2021-03-13 00:00:00  done.\n",
      "2020-06-16 00:00:00  done.\n",
      "2017-09-19 00:00:00  done.\n",
      "2022-04-10 00:00:00  done.\n",
      "2018-03-07 00:00:00  done.\n",
      "2018-07-13 00:00:00  done.\n",
      "2015-12-03 00:00:00  done.\n",
      "2014-05-15 00:00:00  done.\n",
      "2022-06-03 00:00:00  done.\n",
      "2022-08-23 00:00:00  done.\n",
      "2017-05-05 00:00:00  done.\n",
      "2021-05-18 00:00:00  done.\n",
      "2017-01-29 00:00:00  done.\n",
      "2020-06-14 00:00:00  done.\n",
      "2018-04-24 00:00:00  done.\n",
      "2018-12-02 00:00:00  done.\n",
      "2022-05-02 00:00:00  done.\n",
      "2018-03-23 00:00:00  done.\n",
      "2014-06-21 00:00:00  done.\n",
      "2017-09-03 00:00:00  done.\n",
      "2021-09-30 00:00:00  done.\n",
      "2018-06-25 00:00:00  done.\n",
      "2014-11-05 00:00:00  done.\n",
      "2019-01-05 00:00:00  done.\n",
      "2015-06-26 00:00:00  done.\n",
      "2022-07-06 00:00:00  done.\n",
      "2022-01-18 00:00:00  done.\n",
      "2022-02-27 00:00:00  done.\n",
      "2017-08-27 00:00:00  done.\n",
      "2015-10-16 00:00:00  done.\n",
      "2020-05-22 00:00:00  done.\n",
      "2020-03-10 00:00:00  done.\n",
      "2017-10-19 00:00:00  done.\n",
      "2015-11-15 00:00:00  done.\n",
      "2016-08-22 00:00:00  done.\n",
      "2021-06-17 00:00:00  done.\n",
      "2016-04-23 00:00:00  done.\n",
      "2021-09-21 00:00:00  done.\n",
      "2020-04-04 00:00:00  done.\n",
      "2014-09-27 00:00:00  done.\n",
      "2016-10-18 00:00:00  done.\n",
      "2022-10-27 00:00:00  done.\n",
      "2018-06-09 00:00:00  done.\n",
      "2019-07-07 00:00:00  done.\n",
      "2021-02-11 00:00:00  done.\n",
      "2018-11-09 00:00:00  done.\n",
      "2021-10-16 00:00:00  done.\n",
      "2016-12-19 00:00:00  done.\n",
      "2021-11-10 00:00:00  done.\n",
      "2020-04-11 00:00:00  done.\n",
      "2020-01-06 00:00:00  done.\n",
      "2021-12-03 00:00:00  done.\n",
      "2022-01-27 00:00:00  done.\n",
      "2022-02-21 00:00:00  done.\n",
      "2022-03-24 00:00:00  done.\n",
      "2019-05-27 00:00:00  done.\n",
      "2020-12-09 00:00:00  done.\n",
      "2022-02-11 00:00:00  done.\n",
      "2018-12-27 00:00:00  done.\n",
      "2022-05-11 00:00:00  done.\n",
      "2017-06-13 00:00:00  done.\n",
      "2019-05-11 00:00:00  done.\n",
      "2019-04-11 00:00:00  done.\n",
      "2016-01-29 00:00:00  done.\n",
      "2022-06-14 00:00:00  done.\n",
      "2020-12-16 00:00:00  done.\n",
      "2017-12-06 00:00:00  done.\n",
      "2022-11-21 00:00:00  done.\n",
      "2014-08-26 00:00:00  done.\n",
      "2019-12-23 00:00:00  done.\n",
      "2017-11-13 00:00:00  done.\n",
      "2020-03-03 00:00:00  done.\n",
      "2020-10-11 00:00:00  done.\n",
      "2018-05-01 00:00:00  done.\n",
      "2022-12-28 00:00:00  done.\n",
      "2015-02-02 00:00:00  done.\n",
      "2019-11-05 00:00:00  done.\n",
      "2018-10-15 00:00:00  done.\n",
      "2022-12-23 00:00:00  done.\n",
      "2018-01-18 00:00:00  done.\n",
      "2019-06-28 00:00:00  done.\n",
      "2015-04-16 00:00:00  done.\n",
      "2017-11-04 00:00:00  done.\n",
      "2020-02-07 00:00:00  done.\n",
      "2022-10-12 00:00:00  done.\n",
      "2017-03-27 00:00:00  done.\n",
      "2014-06-05 00:00:00  done.\n",
      "2016-12-30 00:00:00  done.\n",
      "2018-02-08 00:00:00  done.\n",
      "2017-07-08 00:00:00  done.\n",
      "2021-04-16 00:00:00  done.\n",
      "2015-08-04 00:00:00  done.\n",
      "2016-07-07 00:00:00  done.\n",
      "2020-04-20 00:00:00  done.\n",
      "2021-08-29 00:00:00  done.\n",
      "2018-02-19 00:00:00  done.\n",
      "2022-11-13 00:00:00  done.\n",
      "2014-07-23 00:00:00  done.\n",
      "2020-11-30 00:00:00  done.\n",
      "2017-11-15 00:00:00  done.\n",
      "2022-11-29 00:00:00  done.\n",
      "2022-04-11 00:00:00  done.\n",
      "2021-11-01 00:00:00  done.\n",
      "2022-04-27 00:00:00  done.\n",
      "2016-06-28 00:00:00  done.\n",
      "2017-07-17 00:00:00  done.\n",
      "2014-09-11 00:00:00  done.\n",
      "2019-04-09 00:00:00  done.\n",
      "2022-05-04 00:00:00  done.\n",
      "2018-02-10 00:00:00  done.\n",
      "2015-01-31 00:00:00  done.\n",
      "2022-07-07 00:00:00  done.\n",
      "2018-04-15 00:00:00  done.\n",
      "2019-06-05 00:00:00  done.\n",
      "2017-09-12 00:00:00  done.\n",
      "2021-04-14 00:00:00  done.\n",
      "2018-05-10 00:00:00  done.\n",
      "2018-01-16 00:00:00  done.\n",
      "2021-11-17 00:00:00  done.\n",
      "2022-10-26 00:00:00  done.\n",
      "2015-05-23 00:00:00  done.\n",
      "2018-03-21 00:00:00  done.\n",
      "2020-12-30 00:00:00  done.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2014-08-17 00:00:00  done.\n",
      "2019-04-16 00:00:00  done.\n",
      "2018-01-09 00:00:00  done.\n",
      "2022-06-30 00:00:00  done.\n",
      "2017-03-25 00:00:00  done.\n",
      "2015-05-02 00:00:00  done.\n",
      "2014-12-14 00:00:00  done.\n",
      "2022-01-28 00:00:00  done.\n",
      "2019-07-14 00:00:00  done.\n",
      "2016-05-20 00:00:00  done.\n",
      "2021-12-04 00:00:00  done.\n",
      "2016-09-25 00:00:00  done.\n",
      "2017-04-03 00:00:00  done.\n",
      "2015-02-18 00:00:00  done.\n",
      "2019-04-27 00:00:00  done.\n",
      "2016-05-18 00:00:00  done.\n",
      "2020-04-02 00:00:00  done.\n",
      "2018-12-04 00:00:00  done.\n"
     ]
    }
   ],
   "source": [
    "# datewise arguments for turkana - pulling zscore and climate \n",
    "args = [(date, pts) for date in valid_dates]\n",
    "\n",
    "# arguments = args\n",
    "# # PARALLEL \n",
    "# if __name__ == '__main__':\n",
    "#     print (\"in main.\")\n",
    "#     with Pool(14) as pool:\n",
    "#         print (\"in pool.\")\n",
    "#         date_result = pool.map(get_date_df, arguments)\n",
    "#         pool.close()\n",
    "\n",
    "date_result = []\n",
    "for arg in args: \n",
    "    date_result.append(get_date_df(arg))\n",
    "    print (arg[0], \" done.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "41bc192c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# make a dataframe from date results \n",
    "dynamic = pd.concat(date_result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "bf65f950",
   "metadata": {},
   "outputs": [],
   "source": [
    "# put into dask dataframe to handle size\n",
    "from dask import dataframe as dd\n",
    "dyn_dd = dd.from_pandas(dynamic, npartitions = 400)\n",
    "# send to parquet file storage \n",
    "name_function = lambda x: f\"Kaffrine_clim-{x}.parquet\"\n",
    "dyn_dd.to_parquet('F:\\\\Senegal_Veg_Model\\\\parquet', name_function=name_function)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e59b2672",
   "metadata": {},
   "source": [
    "### Merge Static With Dynamics using Dask "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "37dc8de8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# static data \n",
    "stat = dd.read_csv('F:\\\\Senegal_Veg_Model\\\\Static\\\\Kaffrine_crop_static_data100m.csv').drop(\"Unnamed: 0\", axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c3f73ab4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# dynamic data\n",
    "path = 'F:\\\\Senegal_Veg_Model\\\\parquet'\n",
    "files = [os.path.join(path, file) for file in os.listdir(path) if 'clim' in file]\n",
    "df = dd.read_parquet(files)\n",
    "clean = df.dropna(how = 'any')\n",
    "\n",
    "files_evi = [os.path.join(path, file) for file in os.listdir(path) if '_EVI' in file]\n",
    "df_evi = dd.read_parquet(files_evi)\n",
    "clean_evi = df_evi.dropna(how = 'any')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d6e86700",
   "metadata": {},
   "outputs": [],
   "source": [
    "# reset index to be the date, then partition by the date so each partition is one date \n",
    "by_date = clean_evi.set_index('date', divisions=sorted(valid_dates), compute = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ca6ef6cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "z_by_date = clean.set_index('date', divisions=sorted(valid_dates), compute = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "73d2d8eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# merge savi to dynamic \n",
    "dyn = dd.merge(by_date, z_by_date, left_on=['lat', 'lon', 'date'], right_on=['lat', 'lon', 'date'])\n",
    "# reset index so you dont lose date information \n",
    "dyn_reset = dyn.reset_index()\n",
    "# merge static to dynamic via FID or indices \n",
    "all_df = dd.merge(dyn_reset, stat, left_on=['lat', 'lon'], right_on=['lat', 'lon'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "79dde71a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# send to parquet file storage \n",
    "name_function = lambda x: f\"kaffrine_all_EVI-{x}.parquet\"\n",
    "all_df.to_parquet('F:\\\\Senegal_Veg_Model\\\\parquet', name_function=name_function)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b61686da",
   "metadata": {},
   "source": [
    "### Get z score of climate variables "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ed4ee607",
   "metadata": {},
   "outputs": [],
   "source": [
    "# read in files\n",
    "path = \"F:\\\\Senegal_Veg_Model\\\\parquet\"\n",
    "files = [os.path.join(path, file) for file in os.listdir(path) if 'kaffrine_all_EVI' in file]\n",
    "data = dd.read_parquet(files) \n",
    "data_full = data.compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "532fa398",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create year and month columns \n",
    "data_full[\"year\"] = data_full.date.dt.year\n",
    "data_full[\"month\"] = data_full.date.dt.month\n",
    "\n",
    "# create year and month columns \n",
    "data[\"year\"] = data.date.dt.year\n",
    "data[\"month\"] = data.date.dt.month"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "2e9438a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# get each location's monthly mean climate variables\n",
    "clim_month_mean = data.groupby(['FID', 'month']).mean().reset_index()[['month', 'FID', '16_day_sum_ppt',\n",
    "       '32_day_sum_ppt', '48_day_sum_ppt', '64_day_sum_ppt', '16_day_mean_temp', '32_day_mean_temp',\n",
    "       '48_day_mean_temp', '64_day_mean_temp']].compute()\n",
    "\n",
    "\n",
    "# get each location's monthly stdv of climate variables\n",
    "clim_month_stdv = data.groupby(['FID', 'month']).mean().reset_index()[['month', 'FID', '16_day_sum_ppt',\n",
    "       '32_day_sum_ppt', '48_day_sum_ppt', '64_day_sum_ppt', '16_day_mean_temp', '32_day_mean_temp',\n",
    "       '48_day_mean_temp', '64_day_mean_temp']].compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "44185cab",
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculate z score of climate variables for turkana and narok based on month and FID specific means and stdvs \n",
    "# merge together the means and stdvs to regular dfs\n",
    "merged1 = data_full.merge(clim_month_mean, how = \"left\", on = [\"FID\", \"month\"])\n",
    "merged = merged1.merge(clim_month_stdv, how = \"left\", on = [\"FID\", \"month\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "17f3cb27",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create new climate variables that are z scores for that point, given the monthly means for that point \n",
    "merged[\"16_day_sum_ppt_Z\"] = (merged[\"16_day_sum_ppt_x\"] - merged[\"16_day_sum_ppt_y\"]) / merged[\"16_day_sum_ppt\"]\n",
    "merged[\"32_day_sum_ppt_Z\"] = (merged[\"32_day_sum_ppt_x\"] - merged[\"32_day_sum_ppt_y\"]) / merged[\"32_day_sum_ppt\"]\n",
    "merged[\"48_day_sum_ppt_Z\"] = (merged[\"48_day_sum_ppt_x\"] - merged[\"48_day_sum_ppt_y\"]) / merged[\"48_day_sum_ppt\"]\n",
    "merged[\"64_day_sum_ppt_Z\"] = (merged[\"64_day_sum_ppt_x\"] - merged[\"64_day_sum_ppt_y\"]) / merged[\"64_day_sum_ppt\"]\n",
    "merged[\"16_day_mean_temp_Z\"] = (merged[\"16_day_mean_temp_x\"] - merged[\"16_day_mean_temp_y\"]) / merged[\"16_day_mean_temp\"]\n",
    "merged[\"32_day_mean_temp_Z\"] = (merged[\"32_day_mean_temp_x\"] - merged[\"32_day_mean_temp_y\"]) / merged[\"32_day_mean_temp\"]\n",
    "merged[\"48_day_mean_temp_Z\"] = (merged[\"48_day_mean_temp_x\"] - merged[\"48_day_mean_temp_y\"]) / merged[\"48_day_mean_temp\"]\n",
    "merged[\"64_day_mean_temp_Z\"] = (merged[\"64_day_mean_temp_x\"] - merged[\"64_day_mean_temp_y\"]) / merged[\"64_day_mean_temp\"]\n",
    "z_clim = merged.drop(['16_day_sum_ppt_y', '32_day_sum_ppt_y', '48_day_sum_ppt_y',\n",
    "   '64_day_sum_ppt_y', '16_day_mean_temp_y', '32_day_mean_temp_y',\n",
    "   '48_day_mean_temp_y', '64_day_mean_temp_y', '16_day_sum_ppt',\n",
    "   '32_day_sum_ppt', '48_day_sum_ppt', '64_day_sum_ppt',\n",
    "   '16_day_mean_temp', '32_day_mean_temp', '48_day_mean_temp',\n",
    "   '64_day_mean_temp', '16_day_mean_ppt',\n",
    "       '32_day_mean_ppt', '48_day_mean_ppt', '64_day_mean_ppt'], axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "e41e9f7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# write to parquet file\n",
    "# repartition so size of files is manageable\n",
    "# df = z_clim.repartition(partition_size = '1MB')\n",
    "\n",
    "# put into dask dataframe to handle size\n",
    "from dask import dataframe as dd\n",
    "df = dd.from_pandas(z_clim, npartitions = 400)\n",
    "# df = z_clim\n",
    "# specify parquet path\n",
    "parquet_path = \"F:\\\\Senegal_Veg_Model\\\\parquet\"\n",
    "# get name function based on lulc \n",
    "name_function = lambda x: f\"kaffrine_crop-{x}.parquet\"\n",
    "# map to new parquet files \n",
    "df.to_parquet(parquet_path, name_function=name_function)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9e253d9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
